# -*- coding: utf-8 -*-
import scrapy
import json,re,datetime
from urllib import parse
from scrapy.loader import ItemLoader
from ArticleSpider.items import ZhihuQuestionItem,ZhihuAnswerItem
class ZhihuSpider(scrapy.Spider):
    name = 'zhihu'
    custom_settings={
        "DOWNLOAD_DELAY":3,
        "COOKIES_ENABLED":True#必须开启cookie不然无法访问
    }
    allowed_domains = ['www.zhihu.com/']
    start_urls = ['https://www.zhihu.com/']
    #question的answer起始url
    start_answer_urls = 'https://www.zhihu.com/api/v4/questions/{0}/answers?include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Crelevant_info%2Cquestion%2Cexcerpt%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&limit={1}&offset={2}&sort_by=default'
    header = {
        "HOST": "www.zhihu.com",
        #"Referer": "https://www.zhihu.com",
    }
    # 使用什么值得买作为实例
    def parse(self, response):
        """
        提取出html页面中所有的url，并跟踪这些url进一步爬取
        如果提取的url中格式为/question/xxx 就下载之后直接进入解析函数
        """
        all_urls=response.css("a::attr(href)").extract()
        all_urls=[parse.urljoin(response.url,url) for url in all_urls]
        #all_urls=filter(lambda x:True if x.startswith("https") else False,all_urls)#https://www.cnblogs.com/jydeng/p/4145188.html
        for url in all_urls:
            match_obj=re.match("(.*zhihu.com/question/(\d+))(/|$).*",url)
            if match_obj:
                request_url=match_obj.group(1)
                if request_url:
                    #如果提取到question相关页面就进行下载解析
                    yield scrapy.Request(request_url,headers=self.header,callback=self.parse_question,dont_filter=True)
                    #由于框架是异步的，所以调试比较麻烦，可疑使用break来结束循环然后既可以在回调中使用断点
                    #break
                else:
                    #如果不是question页面，就继续跟踪
                    yield scrapy.Request(url,headers=self.header,dont_filter=True)

    #页面下载器
    def parse_question(self,response):
        print(response)
        match_obj = re.match("(.*zhihu.com/question/(\d+))(/|$).*", response.url)
        if match_obj:
            question_id = int(match_obj.group(2))
        item_loader = ItemLoader(item=ZhihuQuestionItem(),response=response)
        item_loader.add_css('title','h1.QuestionHeader-title::text')
        item_loader.add_css('content','.QuestionHeader-detail')
        item_loader.add_value('url',response.url)
        item_loader.add_value('zhihu_id',question_id)
        item_loader.add_css('answer_num','h4.List-headerText span::text')
        item_loader.add_css('comments_num','.QuestionHeader-Comment button::text')
        item_loader.add_css('watch_user_num','.NumberBoard-itemInner .NumberBoard-itemValue::text')
        item_loader.add_css('topics','.QuestionHeader-tags a div div::text')
        question_item =item_loader.load_item()
        yield scrapy.Request(self.start_answer_urls.format(question_id,20,0),headers=self.header,callback=self.parse_answer,dont_filter=True)
        yield question_item

    #处理answer
    def parse_answer(self,response):
        #获取分页信息
        ans_json=json.loads(response.text)
        is_end=ans_json['paging']['is_end']
        next_url=ans_json['paging']['next']
        #提取内容
        for answer in ans_json["data"]:
            answer_item = ZhihuAnswerItem()
            answer_item["zhihu_id"] = answer["id"]
            answer_item["url"] = answer["url"]
            answer_item["question_id"] = answer["id"]
            answer_item["author_id"] = answer["author"]["id"] if "id" in answer["author"] else None
            answer_item["content"] = answer["content"] if "content" in answer else None
            answer_item["praise_num"] = answer["voteup_count"]
            answer_item["comments_num"] = answer["comment_count"]
            answer_item["create_time"] = answer["created_time"]
            answer_item["update_time"] = answer["updated_time"]
            answer_item["crawl_time"] = datetime.datetime.now()
            yield answer_item
        if not is_end:
            yield scrapy.Request(next_url, headers=self.header,callback=self.parse_answer, dont_filter=True)
    # 重构Request方法
    def start_requests(self):
        with open("cookies.json", 'r', encoding='utf-8') as json_file:
            cookie = json.load(json_file)
        return [scrapy.Request(
            'https://www.zhihu.com/',
            headers=self.header,
            cookies=cookie,
            dont_filter=True,
            #callback=self.check_login
        )]

    # 由于这里是异步操作，因此如果要处理数据必须指定回调函数
    # 验证登陆成功
    def check_login(self,response):
        for url in self.start_urls:
            yield scrapy.Request(url,headers=self.header,dont_filter=True)
